#!/usr/local/bin/python
#-*- encoding:utf-8 -*-
'''
This script is for the request of extracting low frequency queries from labeled
'''
import random

query_dict = {};
def readOperLog(operlog):
    operlogFile = open(operlog, 'r');
    for line in operlogFile:
        li = line.strip().split('\t');
        queryString = li[2].lower();
        click_count = int(li[1]);
        if query_dict.has_key(queryString):
            query_dict[queryString] += click_count;
        else:
            query_dict[queryString] = click_count;
    operlogFile.close();


def process(labeledSet):
    labeledSetFile = open(labeledSet, 'r');
    temp1  = open('labeled_temp_3', 'w');
    temp2 = open('labeled_temp_4', 'w');
    result_five = open('labeled_week.txt', 'a');
    result_month = open('labeled_month.txt', 'a');
    result_all = open('labeled_all.txt', 'a');
    temp2_list = [(v, k) for k, v in query_dict.items()];

    temp2_list.sort();
    for v, k in temp2_list:
        temp2.write('%d\t%s\n' % (v, k));
    temp2.close();

    labeled = [];
    for line in labeledSetFile:
        li = line.strip().split('|', 1);
        if len(li) != 2:
            print 'Wrong';
            continue;
        queryString = li[0].lower();
        if query_dict.has_key(queryString):
            labeled.append((query_dict[queryString], queryString));
            c = query_dict[queryString];
        else:
            labeled.append((0, queryString));
            c = 0;
        if True:
            if random.random() < 0.5:
                result_five.write('%d\t%s'%(c, line));
            else:
                result_month.write('%d\t%s'%(c,line));
        result_all.write(line);

    labeled.sort();
    result_five.close();
    result_month.close();
    result_all.close();

    for count, queryString in labeled:
        temp1.write('%d\t%s\n' % (count, queryString));

    temp1.close();
    labeledSetFile.close();


if __name__ == '__main__':
    operlog = '/home/wangshuxin/operlog.onemonth';
    labeledSet = 'LabeledData/label_log.txt';
    readOperLog(operlog);
    process(labeledSet);
